{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper functions to build backward context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_binary_ops_tensor(ts1, ts2, grad_fn_ts1, grad_fn_ts2, values):\n",
    "    \"\"\"for binary operator\"\"\"\n",
    "    requires_grad = ts1.requires_grad or ts2.requires_grad\n",
    "    dependency = []\n",
    "    if ts1.requires_grad:\n",
    "        dependency.append(dict(tensor=ts1, grad_fn=grad_fn_ts1))\n",
    "    if ts2.requires_grad:\n",
    "        dependency.append(dict(tensor=ts2, grad_fn=grad_fn_ts2))\n",
    "    tensor_cls = ts1.__class__\n",
    "    return tensor_cls(values, requires_grad, dependency)\n",
    "\n",
    "\n",
    "def build_unary_ops_tensor(ts, grad_fn, values):\n",
    "    \"\"\"for unary operators\"\"\"\n",
    "    requires_grad = ts.requires_grad\n",
    "    dependency = []\n",
    "    if ts.requires_grad:\n",
    "        dependency.append(dict(tensor=ts, grad_fn=grad_fn))\n",
    "    tensor_cls = ts.__class__\n",
    "    return tensor_cls(values, requires_grad, dependency)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Tensor class\n",
    "\n",
    "- needs to define numerical operators\n",
    "- store its dependent tensors \n",
    "- store gradient functions w.r.t its dependent tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def as_tensor(obj):\n",
    "    if not isinstance(obj, Tensor):\n",
    "        obj = Tensor(obj)\n",
    "    return obj\n",
    "\n",
    "\n",
    "class Tensor:\n",
    "    \n",
    "    def __init__(self, values, requires_grad=False, dependency=None):\n",
    "        self._values = np.array(values)\n",
    "        self.shape = self.values.shape\n",
    "        \n",
    "        self.grad = None\n",
    "        if requires_grad:\n",
    "            self.zero_grad()\n",
    "        self.requires_grad = requires_grad\n",
    "        \n",
    "        if dependency is None:\n",
    "            dependency = []\n",
    "        self.dependency = dependency\n",
    "            \n",
    "    @property\n",
    "    def values(self):\n",
    "        return self._values\n",
    "    \n",
    "    @values.setter\n",
    "    def values(self, new_values):\n",
    "        self._values = np.array(new_values)\n",
    "        self.grad = None\n",
    "        \n",
    "    def zero_grad(self):\n",
    "        self.grad = np.zeros(self.shape)\n",
    "        \n",
    "    def __matmul__(self, other):\n",
    "        \"\"\" self @ other \"\"\"\n",
    "        return _matmul(self, as_tensor(other))\n",
    "        \n",
    "    def __rmatmul__(self, other):\n",
    "        \"\"\" other @ self \"\"\"\n",
    "        return _matmul(as_tensor(other), self)\n",
    "    \n",
    "    def __imatmul__(self, other):\n",
    "        \"\"\" self @= other \"\"\"\n",
    "        self.values = self.values @ as_tensor(other).values\n",
    "        return self\n",
    "    \n",
    "    def __add__(self, other):\n",
    "        \"\"\" self + other \"\"\"\n",
    "        return _add(self, as_tensor(other))\n",
    "    \n",
    "    def __radd__(self, other):\n",
    "        \"\"\" other + self \"\"\"\n",
    "        return _add(as_tensor(other), self)\n",
    "    \n",
    "    def __iadd__(self, other):\n",
    "        \"\"\" self += other \"\"\"\n",
    "        self.values = self.values + as_tensor(other).values\n",
    "        return self\n",
    "       \n",
    "    def __sub__(self, other):\n",
    "        \"\"\" self - other \"\"\"\n",
    "        return _sub(self, as_tensor(other))\n",
    "    \n",
    "    def __rsub__(self, other):\n",
    "        \"\"\" other - self \"\"\"\n",
    "        return _add(as_tensor(other), self)\n",
    "    \n",
    "    def __isub__(self, other):\n",
    "        \"\"\" self -= other \"\"\"\n",
    "        self.values = self.values - as_tensor(other).values\n",
    "        return self\n",
    "        \n",
    "    def __mul__(self, other):\n",
    "        \"\"\" self * other \"\"\"\n",
    "        return _mul(self, as_tensor(other))\n",
    "    \n",
    "    def __rmul(self, other):\n",
    "        \"\"\" other * self \"\"\"\n",
    "        return _mul(as_tensor(other), self)\n",
    "    \n",
    "    def __imul(self, other):\n",
    "        \"\"\" self *= other \"\"\"\n",
    "        self.values = self.values * as_tensor(other).values\n",
    "        return self\n",
    "    \n",
    "    def __neg__(self):\n",
    "        \"\"\" -self \"\"\"\n",
    "        return _neg(self)\n",
    "    \n",
    "    def sum(self, axis=None):\n",
    "        return _sum(self, axis=axis)\n",
    "    \n",
    "    \n",
    "    def backward(self, grad=None):\n",
    "        assert self.requires_grad, \"Call backward() on a non-requires-grad tensor.\"\n",
    "        grad = 1.0 if grad is None else grad\n",
    "        grad = np.array(grad)\n",
    "\n",
    "        # accumulate gradient\n",
    "        self.grad += grad\n",
    "\n",
    "        # propagate the gradient to its dependencies\n",
    "        for dep in self.dependency:\n",
    "            grad_for_dep = dep[\"grad_fn\"](grad)\n",
    "            dep[\"tensor\"].backward(grad_for_dep)\n",
    "            \n",
    "            \n",
    "def _matmul(ts1, ts2):\n",
    "    values = ts1.values @ ts2.values\n",
    "\n",
    "    # c = a @ b\n",
    "    # D_c / D_a = grad @ b.T\n",
    "    # D_c / D_b = a.T @ grad\n",
    "    def grad_fn_ts1(grad):\n",
    "        return grad @ ts2.values.T\n",
    "\n",
    "    def grad_fn_ts2(grad):\n",
    "        return ts1.values.T @ grad\n",
    "\n",
    "    return build_binary_ops_tensor(\n",
    "        ts1, ts2, grad_fn_ts1, grad_fn_ts2, values)\n",
    "\n",
    "\n",
    "def _add(ts1, ts2):\n",
    "    values = ts1.values + ts2.values\n",
    "\n",
    "    # c = a + b\n",
    "    # D_c / D_a = 1.0\n",
    "    # D_c / D_b = 1.0\n",
    "    def grad_fn_ts1(grad):\n",
    "        # handle broadcasting (5, 3) + (3,) -> (5, 3)\n",
    "        for _ in range(grad.ndim - ts1.values.ndim):\n",
    "            grad = grad.sum(axis=0)\n",
    "        # handle broadcasting (5, 3) + (1, 3) -> (5, 3)\n",
    "        for i, dim in enumerate(ts1.shape):\n",
    "            if dim == 1:\n",
    "                grad = grad.sum(axis=i, keepdims=True)\n",
    "        return grad\n",
    "\n",
    "    def grad_fn_ts2(grad):\n",
    "        for _ in range(grad.ndim - ts2.values.ndim):\n",
    "            grad = grad.sum(axis=0)\n",
    "        for i, dim in enumerate(ts2.shape):\n",
    "            if dim == 1:\n",
    "                grad = grad.sum(axis=i, keepdims=True)\n",
    "        return grad\n",
    "\n",
    "    return build_binary_ops_tensor(\n",
    "        ts1, ts2, grad_fn_ts1, grad_fn_ts2, values)\n",
    "\n",
    "\n",
    "def _sub(ts1, ts2):\n",
    "    return ts1 + (-ts2)\n",
    "\n",
    "\n",
    "def _mul(ts1, ts2):\n",
    "    values = ts1.values * ts2.values\n",
    "\n",
    "    # c = a * b\n",
    "    # D_c / D_a = b\n",
    "    # D_c / D_b = a\n",
    "    def grad_fn_ts1(grad):\n",
    "        grad = grad * ts2.values\n",
    "        for _ in range(grad.ndim - ts1.values.ndim):\n",
    "            grad = grad.sum(axis=0)\n",
    "        for i, dim in enumerate(ts1.shape):\n",
    "            if dim == 1:\n",
    "                grad = grad.sum(axis=i, keepdims=True)\n",
    "        return grad\n",
    "\n",
    "    def grad_fn_ts2(grad):\n",
    "        grad = grad * ts1.values\n",
    "        for _ in range(grad.ndim - ts2.values.ndim):\n",
    "            grad = grad.sum(axis=0)\n",
    "        for i, dim in enumerate(ts2.shape):\n",
    "            if dim == 1:\n",
    "                grad = grad.sum(axis=i, keepdims=True)\n",
    "        return grad\n",
    "\n",
    "    return build_binary_ops_tensor(\n",
    "        ts1, ts2, grad_fn_ts1, grad_fn_ts2, values)\n",
    "\n",
    "\n",
    "def _neg(ts):\n",
    "    values = -ts.values\n",
    "\n",
    "    def grad_fn(grad):\n",
    "        return -grad\n",
    "\n",
    "    return build_unary_ops_tensor(ts, grad_fn, values)\n",
    "\n",
    "\n",
    "def _sum(ts, axis):\n",
    "    values = ts.values.sum(axis=axis)\n",
    "    if axis is not None:\n",
    "        repeat = ts.values.shape[axis]\n",
    "\n",
    "    def grad_fn(grad):\n",
    "        if axis is None:\n",
    "            grad = grad * np.ones_like(ts.values)\n",
    "        else:\n",
    "            grad = np.expand_dims(grad, axis)\n",
    "            grad = np.repeat(grad, repeat, axis)\n",
    "        return grad\n",
    "\n",
    "    return build_unary_ops_tensor(ts, grad_fn, values)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear regression example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch-0 \tloss: 12002.3536\n",
      "epoch-10 \tloss: 2967.2402\n",
      "epoch-20 \tloss: 759.4492\n",
      "epoch-30 \tloss: 201.3600\n",
      "epoch-40 \tloss: 55.2152\n",
      "epoch-50 \tloss: 15.6038\n",
      "epoch-60 \tloss: 4.5236\n",
      "epoch-70 \tloss: 1.3388\n",
      "epoch-80 \tloss: 0.4027\n",
      "epoch-90 \tloss: 0.1226\n",
      "epoch-100 \tloss: 0.0377\n"
     ]
    }
   ],
   "source": [
    "# training data\n",
    "x = Tensor(np.random.normal(0, 1.0, (100, 3)))\n",
    "coef = Tensor(np.random.randint(0, 10, (3,)))\n",
    "y = x * coef - 3 \n",
    "\n",
    "params = {\n",
    "    \"w\": Tensor(np.random.normal(0, 1.0, (3, 3)), requires_grad=True),\n",
    "    \"b\": Tensor(np.random.normal(0, 1.0, 3), requires_grad=True)\n",
    "}\n",
    "\n",
    "learng_rate = 3e-4\n",
    "loss_list = []\n",
    "for e in range(101):\n",
    "    # set gradient to zero\n",
    "    for param in params.values():\n",
    "        param.zero_grad()\n",
    "    \n",
    "    # forward\n",
    "    predicted = x @ params[\"w\"] + params[\"b\"]\n",
    "    err = predicted - y\n",
    "    loss = (err * err).sum()\n",
    "    \n",
    "    # backward automatically\n",
    "    loss.backward()\n",
    "    \n",
    "    # updata parameters\n",
    "    for param in params.values():\n",
    "        param -= learng_rate * param.grad\n",
    "        \n",
    "    loss_list.append(loss.values)\n",
    "    if e % 10 == 0:\n",
    "        print(\"epoch-%i \\tloss: %.4f\" % (e, loss.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'loss')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfgAAAE5CAYAAAB8nDJuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5Sc9X3f8fd3Zu8rabXalVZXJAECgQwYkA0yEITAOGBM7bZ2XBs7jn1CkyYpdk/t1mlO67ZJGydpE9t1aquua8f3tk6DDbExBhZzv8jc71ddELojrVaX3dXur3/sSEhCRrM7s/PMzL5f5+yZZ37Ps8989QXpM889UkpIkqT6ksu6AEmSVH4GvCRJdciAlySpDhnwkiTVIQNekqQ6ZMBLklSHGrIuoJy6u7vTokWLyra+PXv20N7eXrb1TVb2sXT2sHT2sHT2sHQT0cM1a9ZsSynNPHq8rgJ+0aJFPPjgg2VbX29vLytXrizb+iYr+1g6e1g6e1g6e1i6iehhRKw91ri76CVJqkMGvCRJdciAlySpDhnwkiTVIQNekqQ6ZMBLklSHDHhJkurQhAR8RDRGxI8L0xER34yIeyPiRxHREBEtEXFDRDwSEd8qLFPU2ETUK0lSvSl7wEdEK7AGeGdh6AKgIaV0PjANuBy4BtiQUjoL6CwsW+yYJEk6jrIHfEppX0rpTGBDYWgz8IXC9GDhdRVwc2H6VuCSMYxVxKu79nHruiG29w9U6iMlSSqbCb9VbUrpOYCIeB/QBNwEXAfsKizSB5wKdBU5doSIuBa4FqCnp4fe3t6y1P3U9mH+5slBZrffyeld+bKsc7Lq7+8v23+Xycoels4els4elq6SPazIvegj4mpGQ/09KaXhiNgGdBRmdwDbgClFjh0hpbQaWA2wfPnyVK57/J60Yy+ff+A2uk5Ywsq3nVCWdU5W3r+6dPawdPawdPawdJXs4YSfRR8Rs4FPA+9OKe0uDN/C6LF4GN0Nf9sYxipiTkcLuYANr+2r1EdKklQ2lbhM7jeBOcBNEXFnRHwc+A4wLyIeBXYwGuTFjlVEQz5HZ3MY8JKkmjRhu+hTSicXXj8PfP4Yi1x11PuBIscqprs12PDa3qw+XpKkcfNGN2+iuzXnFrwkqSYZ8G+iuzXY1LefwQMjWZciSdKYGPBvors1SGn0mnhJkmqJAf8multH2/OKu+klSTXGgH8T3a2jt773OLwkqdYY8G+isyUK18J7Jr0kqbYY8G+iIRfM6Wh1C16SVHMM+OOY12nAS5JqjwF/HPM7W91FL0mqOQb8cczvbPNaeElSzTHgj2N+ZysjCTbt2p91KZIkFc2AP475na2AZ9JLkmqLAX8cCzrbAK+FlyTVFgP+OGYfei68W/CSpNphwB9HYz7H7GktbsFLkmqKAV+E+Z1tBrwkqaYY8EXwWnhJUq0x4Iswv7PVa+ElSTXFgC/C/M42r4WXJNUUA74Ih66F3+lueklSbTDgizDfa+ElSTXGgC/C69fCG/CSpNpgwBehqeHgtfDuopck1QYDvkheCy9JqiUGfJHmd7byigEvSaoRBnyR5ne28uqufQwNey28JKn6GfBF8lp4SVItMeCLdPBa+PU7PNFOklT9DPgiLZgxei38WgNeklQDDPgizZ3eSlM+x8vb92RdiiRJx2XAFymfCxbMaOXlbQa8JKn6GfBjsLi7nZe3uYteklT9DPgxWNTVzsvb9zAykrIuRZKkNzUhAR8RjRHx48J0S0TcEBGPRMS3YtS4xyai3mIt6m5n4MAIm/q8VE6SVN3KHvAR0QqsAd5ZGLoG2JBSOgvoLIyXMpaZxd3tAB6HlyRVvbIHfEppX0rpTGBDYWgVcHNh+lbgkhLHMrPoYMBv9zi8JKm6VeIYfBewqzDdB8wocSwzc6a10NTgpXKSpOrXUIHP2AZ0FKY7Cu+nlDB2hIi4FrgWoKenh97e3rIV3t/f/4b1dbckHnh6Lb1tm8v2OfXuWH3U2NjD0tnD0tnD0lWyh5UI+FuAy4EfMrrL/S+BE0oYO0JKaTWwGmD58uVp5cqVZSu8t7eXo9e3bN2DvLxtDytXXly2z6l3x+qjxsYels4els4elq6SPazELvrvAPMi4lFgB6OBX8pYphZ3t7N2x14vlZMkVbUJ24JPKZ1ceB0ArjpqdiljmVrU1c7ggRE27trH/M62rMuRJOmYvNHNGC3qLjx0xjPpJUlVzIAfo4PXwr/ktfCSpCpmwI9Rz9QWmhty3uxGklTVDPgxyuXi0D3pJUmqVgb8OCzqbnMXvSSpqhnw47Cou531O/Yx7KVykqQqZcCPw+KudgaHR9i4c1/WpUiSdEwG/Di8/tAZd9NLkqqTAT8OPjZWklTtDPhxmDW1mdbGPC9t82Y3kqTqZMCPQ0SwsKvNXfSSpKplwI/T4m6vhZckVS8DfpxGL5Xby4HhkaxLkSTpDQz4cVrc1c7QcGLjzv1ZlyJJ0hsY8ON08FK5l9xNL0mqQgb8OJ04czTgX9jSn3ElkiS9kQE/Tl3tTXS2NfKcAS9JqkIG/DhFBEtmTeW5zbuzLkWSpDcw4EuwpGcKz23pJyUfOiNJqi4GfAmWzJrCrn1DbN09kHUpkiQdwYAvwSk9UwE8Di9JqjoGfAlO7pkCwLMeh5ckVRkDvgQzpzQz3TPpJUlVyIAvweiZ9FM8k16SVHUM+BIt6ZnKs5s9k16SVF0M+BIdOpO+3zPpJUnVw4Av0cEz6Z/f7HF4SVL1MOBLtGSWZ9JLkqqPAV+imVOb6Wj1THpJUnUx4Ev0+pn0BrwkqXoY8GWwpGcqz27Z7Zn0kqSqYcCXwZJZU9i5d4ht/YNZlyJJEmDAl8WSwi1rn9viiXaSpOpQkYCPiPaIuD4i7oqIP4uI7oi4IyIei4g/LSxT1Fg1OvTQGY/DS5KqRKW24D8M3JtSugBYBnwVuBE4C7giIk4BPlnkWNWZNbWZqS0NbsFLkqpGpQJ+AGiLiABagHcAN6eURoDbgUuAVUWOVZ2I4JTCLWslSaoGlQr47wJXAE8BTwN9wK7CvD5gBtBV5FhVWjJrCs97LbwkqUo0VOhzPgt8JaX0tYj4HnAK0FGY1wGsBbYVOXaEiLgWuBagp6eH3t7eshXd399f9Ppi9xA79gzyo5tuY1pzlK2GejCWPurY7GHp7GHp7GHpKtnDSgX8VGB/YXoAuAe4PCIeAi4GvgAsKHLsCCml1cBqgOXLl6eVK1eWreje3l6KXV/u2a187+n7mXnymaw4qatsNdSDsfRRx2YPS2cPS2cPS1fJHlZqF/2Xgd+NiHuAVuB9wJXAo8CNKaXngS8WOVaVDp5J7z3pJUnVoCJb8Cmll4ELjhq+6KhlthUzVq16pjXT2dbIkxv7si5FkiRvdFMuEcGyuR088equ4y8sSdIEM+DLaNm8aTy7qZ/BAyNZlyJJmuQM+DJaNreDweERb3gjScqcAV9Gy+ZOA+AJj8NLkjJmwJfR4q522prynmgnScqcAV9GuVxw2pxpPLHRE+0kSdky4Mts2dxpPLmxj5GRlHUpkqRJzIAvs2Vzp7FncJi1O/ZmXYokaRIz4Mts2dzRW+e7m16SlCUDvsyW9EyhIReeSS9JypQBX2bNDXmW9Ew14CVJmTLgJ8DoiXa7SMkT7SRJ2TDgJ8CyudPY1j/Ilt0DWZciSZqkDPgJ4Il2kqSsGfAT4LQ5o8+Gf+IVj8NLkrJhwE+AqS2NLOpq80Q7SVJmDPgJ4rPhJUlZMuAnyOlzp7F+xz527RvKuhRJ0iRkwE+Qg4+O9clykqQsGPATxDPpJUlZMuAnyMypzcyb3spD63dmXYokaRIy4CfQ2SdM5+F1BrwkqfIM+Al09gmdvLJzH5v79mddiiRpkjHgJ9A5J0wH4JdrX8u4EknSZGPAT6DT506jKZ/zOLwkqeIM+AnU3JDnLfOmuQUvSao4A36CnX1CJ4+9sovBAyNZlyJJmkQM+Al2zgmdDBwY4alXveGNJKlyDPgJdnbhRLuH1rmbXpJUOQb8BJs7vZXZ01r4pdfDS5IqyICvgHMWTueXbsFLkiqo6ICPiFxETIuIfERcEhFTJ7KwenL2gk42vLaPLbu94Y0kqTLGsgX/feB84C+ATwB/NyEV1aFzFh48Du9ueklSZYwl4OemlH4GnJhSugaYMkE11Z1lcztozIcBL0mqmLEE/I6I+DvgsYi4ChhTWkXEZyLijoj4SUTMKkw/FhF/WpjfXcxYLWppzHP63A6Pw0uSKmYsAf9+4D+klP4I2AB8oNhfjIgTgWUppYuAnwB/BdwInAVcERGnAJ8scqwmnb1gOo9u2MmBYW94I0maeGMJ+CHg+YjIA53AWJLqUqAzIn4BXAQsBm5OKY0AtwOXAKuKHKtJ5yzsZP/QCE9v2p11KZKkSaBhDMt+H/gacAUwE/gjRoO7GDOBrSmlqyPiHuDtwK7CvD5gBtBV5NgRIuJa4FqAnp4eent7x/BHenP9/f1lW9/A3tHvQ9//+f1ctrCxLOusFeXs42RlD0tnD0tnD0tXyR6OJeDnppR+FhG/l1L6BxFx3xh+tw94pjD9IjAL6Ci87wDWAtuKHDtCSmk1sBpg+fLlaeXKlWMo68319vZSrvWllPiLh29hZ+MMVq48pyzrrBXl7ONkZQ9LZw9LZw9LV8keVuokuzXA2wrTJzMa9pdHRA64GLgNuKXIsZoUEaw4sYt7X9xBSinrciRJda4iJ9mllO4BtkXEA4yG+0eBK4FHgRtTSs8DXyxyrGatOKmLbf0DPL+lP+tSJEl1biy76IeBcyPiI8ATwONj+aCU0u8eNXTRUfO3FTNWy1ac2A3A3S9sZ0mPNwKUJE2csWzBfwOYB/y08PqNCainri2Y0cq86a3c88L2rEuRJNW5sWzBLyzcwQ7gpoi4cyIKqmcRwYqTuvj5U5sZGUnkcpF1SZKkOjWWLfj1EfFvImJVRPwhsG6iiqpnK07sYufeIa+HlyRNqLEE/McYPXP+HxVePzYB9dS9FSd1AXDPi+6mlyRNnKJ30aeUBoEvT2Atk8Lc6a0s7Grjnhe284kLF2ddjiSpTh034CPiNuDoC7cDSCmlVRNSVZ1bcWIXNz72KsMjibzH4SVJE+C4AZ9Sqtn7v1erFSd18f0H1vPExl2cOX961uVIkurQWI7Bq0xWnFg4Du/lcpKkCWLAZ2DWtBZOmtnuiXaSpAljwGdkxUldPPDSDoZ8PrwkaQIY8Bl5x0nd7Bkc5rFXdh1/YUmSxsiAz8j5HoeXJE0gAz4jM9qbOH3ONG5/dmvWpUiS6pABn6FVS2exZu1r7No7lHUpkqQ6Y8Bn6JKlsxgeSfziObfiJUnlZcBn6K0LpjOjvYnbnt6SdSmSpDpjwGconwsuPmUmtz2zheGRo+8GLEnS+BnwGbtk6Sxe2zvEw+t3Zl2KJKmOGPAZu3jJTPK5cDe9JKmsDPiMdbQ1cu4JndxqwEuSysiArwKXLJ3Fk6/2sWnX/qxLkSTVCQO+CqxaOguA255xK16SVB4GfBU4pWcK86a3upteklQ2BnwViAhWLZ3FXc9vY//QcNblSJLqgAFfJVYtncXewWHue2lH1qVIkuqAAV8lVpzURUtjjluf2px1KZKkOmDAV4mWxjy/tmQmP31iEyPe1U6SVCIDvoq8+8w5bO4b4MG1r2VdiiSpxhnwVeSy03pobshx46Mbsy5FklTjDPgq0t7cwKqls/j7xzf58BlJUkkM+Cpz1Zlz2bp7gPs9m16SVAIDvspcsnQmrY15bnA3vSSpBAZ8lWlrauDS02bx08c3cWB4JOtyJEk1qqIBHxGfioifR0R3RNwREY9FxJ8W5hU1NhlcdeYctu8Z5N4X3U0vSRqfigV8RCwEPlZ4+0ngRuAs4IqIOGUMY3Vv5amzaG/Kc+Nj7qaXJI1PJbfgvwB8tjC9Crg5pTQC3A5cMoaxutfSmOey03v4yeObGHI3vSRpHBoq8SER8SHgEeDJwlAXsKsw3QfMGMPY0eu+FrgWoKenh97e3rLV3d/fX9b1jcWi3AGu3zvEV/72Vs6YWZH/TBMmyz7WC3tYOntYOntYukr2sFLJcRVwAvAu4FRgBOgozOsA1gLbihw7QkppNbAaYPny5WnlypVlK7q3t5dyrm8sVhwY5utP/Jy1zOQPVp6VSQ3lkmUf64U9LJ09LJ09LF0le1iRXfQppQ+llC4EPgisAb4MXB4ROeBi4DbgliLHJoXmhjxXnjGHv3/sVfoHDmRdjiSpxmR1mdwXgSuBR4EbU0rPj2Fs0vjA2xawd3CYGx7xZDtJ0thU9OBuSull4LLC24uOmretmLHJ5JwTprNk1hR+8OB6Pvj2E7IuR5JUQ7zRTRWLCH7jbQt4aN1Ontm0O+tyJEk1xICvcv/wnPk05oMfPLA+61IkSTXEgK9yM9qbuPz02fy/hzYwcGA463IkSTXCgK8Bv/G2Bby2d4ibn9ycdSmSpBphwNeAC0/uZt70VnfTS5KKZsDXgFwu+MDyBdzx3DbW79ibdTmSpBpgwNeIf7x8PhHwf9ZsyLoUSVINMOBrxLzprVx8yky+d/86T7aTJB2XAV9DPn7BYrbuHuDHj7yadSmSpCpnwNeQi5Z0s3T2VL52x4uklLIuR5JUxQz4GhIRfOLCxTy9aTd3Pr8t63IkSVXMgK8xV791LjOnNvO1O17KuhRJUhUz4GtMc0Oe31yxkNuf3cqzm70/vSTp2Az4GvTh8xbS0pjja3e8mHUpkqQqZcDXoM72Jt5/7gL+7qGNbNm9P+tyJElVyICvUR+/cDFDIyN8+561WZciSapCBnyNWtzdzjtP6+Ebd7/Mrn1DWZcjSaoyBnwN++eXLqFv/wH+552eUS9JOpIBX8PeMq+DK8+YzdfvfIkdewazLkeSVEUM+Br3qctOYc/gAb76ixeyLkWSVEUM+Bq3pGcq733rPL5598ueUS9JOsSArwPXXbqEoeHEX9/mVrwkaZQBXwcWdbfz/nPn89371rFx576sy5EkVQEDvk78waVLAPjSrc9lXIkkqRoY8HVi3vRWPnTeCfzggfU89Wpf1uVIkjJmwNeRT162hI7WRv7d9U/4vHhJmuQM+Doyva2Jz/z6Uu5/eQc/emRj1uVIkjJkwNeZDyxfwJnzO/iTG5+if+BA1uVIkjJiwNeZfC7491cvY8vuAb50iyfcSdJkZcDXobNP6OT9587n63e9xPNb+rMuR5KUAQO+Tn3m15fS0pjncz/yhDtJmowM+Do1c2ozn37Xqdz5/Da+/8D6rMuRJFWYAV/HrjlvIStO7OKPb3iS9Tv2Zl2OJKmCKhLwMeqbEXFvRPwoIqZExA0R8UhEfKswv6WYsUrUWy9yueDP338mEcG//D+PMDLirnpJmiwqtQV/AdCQUjofmAZ8HNiQUjoL6ATeCVxT5JjGYH5nG//2Padz30s7+PpdL2VdjiSpQioV8JuBLxSmB4HPATcX3t8KXAKsKnJMY/T+c+dz2Wmz+LObnuH5LbuzLkeSVAENlfiQlNJzABHxPqAJWAPsKszuA04FuoocO0JEXAtcC9DT00Nvb2/Z6u7v7y/r+rJ01ewR7n1+hN/+2p384fktNOYqd7SjnvqYFXtYOntYOntYukr2sCIBDxARVwPXAe8BvgJ0FGZ1ANuAKUWOHSGltBpYDbB8+fK0cuXKstXc29tLOdeXtZZ5r/I73/4lt/d188fvPaNin1tvfcyCPSydPSydPSxdJXtYqZPsZgOfBt6dUtoN3AJcXpi9CrhtDGMap19/yxyu/bUT+fa96/i/azZkXY4kaQJV6hj8bwJzgJsi4k6gEZgXEY8COxgN8u8UOaYSfOZdp7LixC7+zf97jMdf2XX8X5Ak1aRKHYP/PPD5o4a/etT7AeCqIsZUgoZ8ji996Gze86U7+Z1vr+HHv38hne1NWZclSSozb3QzCXVPaeavP3wOW/oGuO4HD3NgeCTrkiRJZWbAT1Jnn9DJv/8Hy/jFs1v5o7973PvVS1KdqdhZ9Ko+/+TtJ7Dhtb18+bYXmDW1mX9x+RuuQpQk1SgDfpL7l5efytbdA3zx1ueZOa2Fj5y/MOuSJEllYMBPchHBf3rfGWzvH+TfXv84M6c08etvmZN1WZKkEnkMXjTkc/y3D53D2Qum88+/9zC3Pr0565IkSSUy4AVAa1Oer3/sbZw6eyr/9Ftr+Onjr2ZdkiSpBAa8Dpne1sR3fvs8zpjXwe999yGuf/iVrEuSJI2TAa8jTGtp5G8+cR7LF3byyR88zP9+YH3WJUmSxsGA1xtMaW7gG7/1di48uZvP/PBRvnTLc14nL0k1xoDXMbU25fkfH13Oe986l/9y87N86gcPs39oOOuyJElF8jI5/UotjXn+8jfeysmzpvAXP3uWdTv28tWPLGfm1OasS5MkHYdb8HpTEcHvr1rCX3/4HJ58tY/3fvkuHlm/M+uyJEnHYcCrKFeeMYf//U9XkFLiH/33u1n9ixcYGfG4vCRVKwNeRTtz/nT+/rqLuPS0Wfynv3+a3/rGA2zrH8i6LEnSMRjwGpPpbU185Zpz+Y/vfQv3vLidK75wBz97YlPWZUmSjmLAa8wigo+cv5Af/f4FdLU3ce231vDPvrOGLX37sy5NklRgwGvcls6exo//4EI+/a5T+flTW7j0v97O9+5f57F5SaoCBrxK0pjP8XuXnMxPr7uIZXOn8dm/fYz3/vVd3Pvi9qxLk6RJzYBXWZw4cwrf++3z+a8fOIutuwf44Op7+e2/eZAXtvZnXZokTUre6EZlExH8w3Pmc+UZc/ifd77Ef+99gcv/8he8Y06ehW/Zw+Lu9qxLlKRJwy14lV1LY57fu+Rkej+9ko+cv5B7Xz3Apf+ll+u+/xDPbt6ddXmSNCm4Ba8J0z2lmc9dvYy3Nm/mqeE5fOvetVz/8EYuOXUmH33HIi5eMpNcLrIuU5LqkgGvCTe9OcdnV57G71x8Et+4+2W+e/86fut/PcCirjauOX8h//jc+Uxva8q6TEmqK+6iV8V0tjfxqXeewl3/ahVf/Cdn0z2lmT++8Sne/ie38LvfXsPNT25maHgk6zIlqS64Ba+Ka2rIcfVZc7n6rLk8ubGPH/5yA9c//Ao/eXwTM9qbuOIts7niLXM478QZNOb9DipJ42HAK1Onz53G6XNP519fsZQ7ntvKD3/5Cn/7y1f4zn3r6Ght5LLTenjn6T1ccHIXU1sasy5XkmqGAa+q0JjPsWppD6uW9rB/aJhfPLuVnz6+iZuf3MQPf7mBhlxwzsJOLj5lJhee3M2yudNocOtekn4lA15Vp6Uxz+XLZnP5stkMDY/w0Lqd3P7sFm5/dit/ftMz/PlNz9DelOfcRTM4b/EM3rZoBmfM66C1KZ916ZJUNQx4VbXGfI63L57B2xfP4NPvWsrW3QPc++J27n9pB/e9tJ0/v+kZAPK54NSeqZy1YDpnze/g9LnTOKVnKi2Nhr6kycmAV02ZObWZ95w1l/ecNReA7f0DPLx+56GfGx7dyPfuXweMhv7JM6ewdM5UlsyawsmzpnDyrKks7Grz5D1Jdc+AV03rmtLMpaf1cOlpPQCMjCTW7djLU6/28eSrfTyxsY8HX36N6x/eeOh3GnLBghltLOxqY1FXO4u62lgwo415na3M72xjSrN/LSTVPv8lU13J5YJF3e0s6m7nijPmHBrfM3CAF7b289zmfl7Y2s/a7Xt5adse7n9pB3sHh49YR0drI3M6WpjT0cLsjlbmdLQwa2ozM6c2M2tqC7OmNdPZ1kRTg3sBJFWvqg74iGgB/i+wAHgU+GhKyYeNa8zamxs4c/50zpw//YjxlBLb+gd5Zec+1u/Yy4bX9vHKzr1s2rWfV3ft59ENu9i+Z/CY65zW0kDXlGZmtDfR2dbI9LbXX6e1NtJR+JnW0sDUlkamtjQwtaWB1sY8Ed6iV9LEquqAB64BNqSUroqIG4B3Aj/LuCbVkYhgZmHr/K0Lph9zmYEDw2zrH2Tr7gG29O1ny+4BduwZZMeeQbb1D7C9f5BXdu7niY19vLZ3kP1Db343vnwuaGvKM6W5gfbmBtqb8rQ1NdDWlKetuYG2xjytTXlaGvO0NOZobcyzfu0Qm+5fd2isuSFPc0OOpobR6aaGHI35oKkw1pTP0ZjP0ZAPGnM57/kvTULVHvCrgB8Wpm8FLsGAV4U1N+SZN72VedNbi1p+/9AwffuG6Ns/xK59Q/TtO0Df/iH6Bw6we/8Bdu8fYs/AMHsGDrBn8AD9A8PsGzzApr4h9g0Os3dwmH1Doz+DBw77svDUY+P+M+RzQWMh7BvyQUM+R0MuRqdzo9P5wvt8Lkc+oCGXI18YP/iTiyCfg1wEuVyQj9HxCMhHHBrPxehnBqNfonIxOpYrLHvwfTD6ymHvR+dzaC9HLkbHCouRK4xHxKGxg5/DYe85+L4w+dz6ITbet+71+YeWfX3g9fE46v2RrwcFcczxQ/OPmnH0Yr9qfb96/ps7/o6h43/Re7N1PL75AANPbCrxE8qr1vaGPb7lABcMj1TkRN9qD/guYFdhug849egFIuJa4FqAnp4eent7y/bh/f39ZV3fZDXZ+xhAR+EHgObCz6+UL/zASEoMDsNru/fQ1NLG0AgMDieGRuDACAyNjE4ffH9gJBVeYTi9/n44vf5+OCWGR4YZKYwNjySGE4yMwPAwjCQ4kBL7CvNHDnsdSaOHNUYSjADp8HEOf02kw8cKB9YO/x14/bVix92eGP+XJBU8tCbrCmreqZ2309Y48V9Mqj3gt/H6v4sdhfdHSCmtBlYDLF++PK1cubJsH97b20s51zdZ2cfSTYYeHvzicOiV9PoXg8O+MBycz2HLpMPWMbpM4f3oQiTg7rvvZsWKd4yOFZY5uD4O+53X6zlsHceaf9hnHv7+6N9/428ce/7xfj8d52vQ8c5OKubspeN9xoMPPsjy5ctL+ozJ7sEHH+Rdl64kX4HDZtUe8LcAlzO6m34V8JfZliNpokQE+dF96xOy/s6WHLM7WiZk3ZPF1ml5ls3tOP6C+pW2PZevSLhD9T8u9jvAvIh4FNjBaOBLkqTjqOot+JTSAHBV1nVIklRrqn0LXpIkjYMBL/4X75kAAAR1SURBVElSHTLgJUmqQwa8JEl1yICXJKkOGfCSJNUhA16SpDpkwEuSVIeinh6vHhFbgbVlXGU3x7j/vcbMPpbOHpbOHpbOHpZuInq4MKU08+jBugr4couIB1NKv/rJCiqKfSydPSydPSydPSxdJXvoLnpJkuqQAS9JUh0y4N/c6qwLqBP2sXT2sHT2sHT2sHQV66HH4CVJqkNuwUuSVIcM+GOIiJaIuCEiHomIb0VEZF1TrYhR34yIeyPiRxExxV6OT0R8KiJ+HhHdEXFHRDwWEX+adV21JCI+U+jdTyJiln0cm4hoj4jrI+KuiPgz/18cm4hojIgfF6bfkCsTnTUG/LFdA2xIKZ0FdALvzLieWnIB0JBSOh+YBnwcezlmEbEQ+Fjh7SeBG4GzgCsi4pSs6qolEXEisCyldBHwE+CvsI9j9WHg3pTSBcAy4KvYw6JERCuwhtf/zTtWrkxo1hjwx7YKuLkwfStwSYa11JrNwBcK04PA57CX4/EF4LOF6VXAzSmlEeB27GGxLgU6I+IXwEXAYuzjWA0AbYUtyxbgHdjDoqSU9qWUzgQ2FIaOlSsTmjUG/LF1AbsK033AjAxrqSkppedSSvdHxPuAJka/wdrLMYiIDwGPAE8Whvz/cXxmAltTSr8GzAfejn0cq+8CVwBPAU8z2jd7OD7H+ns8oX+3Dfhj2wZ0FKY78NaMYxIRVwPXAe8BtmAvx+oqRrc+vw+cy+itLe3h2PUBzxSmXwRexj6O1WeBr6SUljIaPqdgD8frWLkyoVljwB/bLcDlhelVwG0Z1lJTImI28Gng3Sml3djLMUspfSildCHwQUb3gHwZuDwicsDF2MNirQHeVpg+mdGwt49jMxXYX5geAO7BHo7Xsf4tnNB/Hw34Y/sOMC8iHgV2MPofQcX5TWAOcFNE3Ak0Yi9L9UXgSuBR4MaU0vMZ11MTUkr3ANsi4gFGw/2j2Mex+jLwuxFxD9AKvA97OF7HypUJzRpvdCNJUh1yC16SpDpkwEuSVIcMeEmS6pABL0lSHTLgJU2oiPhcRKzMug5psjHgJUmqQw1ZFyCp+kREG/A3jN7T4GGgndGHB81j9OEj10VEV2GZGcB9KaVPRkQ38E1G7773JPCJwiovi4g/AaYA70opbaroH0iahNyCl3Qs1wKPF54iNofRcP/blNJ5wCkRcQ6jtzH9fkppBaMPdXkX8IfAtwvLPQssLKzvVOBCRu9tvqqyfxRpcjLgJR3LqcD7IqIXOJHRxwA/UJj3MKNPZjud0VuXUng9HVgK3F8Y+zNgbWH6m2n0rlqbGX0IkaQJZsBLOpZngL9KKa0E/h1wB3BeYd45wAvAE8D5hbHzC++fPmy51by+td4/8SVLOpwBL+lY/gdwZUTczeju+hzw7oi4D3gypfQw8J+BDxbuU74zpfSzwthHIuIuYASfPSBlxnvRSzquiPgG8LmU0ssZlyKpSAa8JEl1yF30kiTVIQNekqQ6ZMBLklSHDHhJkuqQAS9JUh0y4CVJqkP/H6HRQlb6cEgRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 5))\n",
    "plt.plot(loss_list)\n",
    "plt.grid()\n",
    "plt.xlabel(\"epoch\")\n",
    "plt.ylabel(\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
